﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows.Forms;

namespace NewsSpider.BLL
{
    /// <summary>
    /// 版权所有: 版权所有(C) 2012，上海云里网络科技有限公司
    /// 内容摘要: WebSites
    /// 完成日期：2012年2月2日
    /// 版    本：V1.0 
    /// 作    者：Adin
    /// </summary>
    public class WebSites
    {
        /// <summary>
        /// 采集列表页面所需内容块
        /// </summary>
        /// <param name="rtb">消息控件名称</param>
        public static void GetListPageHtml(System.Windows.Forms.RichTextBox rtb)
        {
            string strSiteName = "";
            string strLink = "";
            try
            {
                NewsSpider.Utility.IniFile iniFile = new Utility.IniFile(System.Windows.Forms.Application.StartupPath + "\\config.ini");
                string strListTagStart = iniFile.GetString("CATCH", "ListStartTag", "");
                string strListTagEnd = iniFile.GetString("CATCH", "ListEndTag", "");
                strLink = iniFile.GetString("CATCH", "ListPageUrl", "");
                strSiteName = iniFile.GetString("CATCH", "SiteName", "");
                string strCharset = iniFile.GetString("CATCH", "Charset", "");

                NewsSpider.Utility.WebClient wc = new NewsSpider.Utility.WebClient();
                wc.Encoding = System.Text.Encoding.GetEncoding(strCharset);
                wc.OpenRead(strLink);
                string strGetHtmlCode = wc.RespHtml;

                //采集列表源代码
                Regex rx = new Regex(strListTagStart + "(?<content>[\\s\\S]*)?" + strListTagEnd, RegexOptions.Singleline);
                string strGetListContentHtml = rx.Match(strGetHtmlCode).Result("${content}");

                GetListPageLinks(rtb, strGetListContentHtml);
            }
            catch (Exception ex)
            {
                string strError = "采集列表页面所需内容块失败：" + strSiteName + ",地址：" + strLink + "\r\n" + ex.Message;
                NewsSpider.Utility.Log.SaveErrorLog("采集列表页面所需内容块 -> " + strError);

                NewsSpider.Utility.PublicValue.SaveMessage(Convert.ToDateTime(DateTime.Now).ToString("MM-dd HH:mm:ss") + " 【" + strSiteName + "】采集列表页面所需内容块失败！");
                rtb.Text = NewsSpider.Utility.PublicValue.GetMessage();
            }

        }

        /// <summary>
        /// 采集列表页面所需内容块中的链接
        /// </summary>
        /// <param name="rtb">消息控件名称</param>
        /// <param name="strGetListContentHtml">采集列表页面的内容块</param>
        public static void GetListPageLinks(System.Windows.Forms.RichTextBox rtb,string strGetListContentHtml)
        {
            try
            {
                NewsSpider.Utility.IniFile iniFile = new Utility.IniFile(System.Windows.Forms.Application.StartupPath + "\\config.ini");
                string strLinkStartTag = iniFile.GetString("CATCH", "LinkStartTag", "");
                string strLinkEndTag = iniFile.GetString("CATCH", "LinkEndTag", "");
                string strSiteName = iniFile.GetString("CATCH", "SiteName", "");

                Regex clearString = new Regex(strLinkStartTag + "(.+?)" + strLinkEndTag, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                Match m = null;
                for (m = clearString.Match(strGetListContentHtml); m.Success; m = m.NextMatch())
                {
                    //采集到链接地址
                    string strLink = m.Groups[1].ToString().Trim();
                    strLink = strLink.Replace("\"","");

                    #region

                    NewsSpider.Utility.PublicValue.SaveMessage(Convert.ToDateTime(DateTime.Now).ToString("MM-dd HH:mm:ss") + " 正在采集链接对应的文章 -> " + strLink);
                    rtb.Text = NewsSpider.Utility.PublicValue.GetMessage();

                    //写入数据
                    NewsSpider.Utility.IniFile iniFileLink = new Utility.IniFile(System.Windows.Forms.Application.StartupPath + "\\config.ini");
                    iniFileLink.WriteValue("CATCH", "LINK", strLink);

                    string strStatus = "";
                    bool bValue = false;
                    int iOverTime = int.Parse(iniFile.GetString("CATCH", "OVERTIME", ""));//读取超时时间，默认10秒=100000
                    bValue = NewsSpider.Utility.OverTimeCntrol.CallFuncThread(GetArticlePageContent, TimeSpan.FromMilliseconds(iOverTime), null);

                    if (!bValue)
                    {
                        strStatus = "读取网址超时(" + iOverTime / 1000 + "秒)，跳过！";
                    }
                    else
                    {
                        int iNum = int.Parse(iniFile.GetString("CATCH", "STATUSCODE", ""));

                        if (iNum == 0)
                        {
                            strStatus = "采集成功";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "catch_success");
                        }
                        if (iNum == 1)
                        {
                            strStatus = "文章标题采集失败";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "fail_title");
                        }
                        if (iNum == 2)
                        {
                            strStatus = "文章内容采集失败";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "fail_content");
                        }
                        if (iNum == 3)
                        {
                            strStatus = "文章来源采集失败";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "fail_source");
                        }
                        if (iNum == 4)
                        {
                            strStatus = "文章作者采集失败";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "fail_author");
                        }
                        if (iNum == 5)
                        {
                            strStatus = "文章内容为空";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "content_null");
                        }
                        if (iNum == 6)
                        {
                            strStatus = "文章抓取重复";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "title_repeat");
                            NewsSpider.Utility.PublicValue.SaveMessage(DateTime.Now.ToString("MM-dd HH:mm:ss") + " 列表页面没有更新文章，跳过！");
                            rtb.Text = NewsSpider.Utility.PublicValue.GetMessage();
                            break;
                        }
                        if (iNum == 7)
                        {
                            strStatus = "添加文章至数据库时异常错误";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "exception_insert");
                        }
                        if (iNum == 8)
                        {
                            strStatus = "打开文章页面异常错误";
                            NewsSpider.Utility.Log.SaveHTMLLog(DateTime.Now.ToString() + "," + strSiteName + "," + strLink, "exception_page");
                        }
                    }

                    NewsSpider.Utility.PublicValue.SaveMessage("\r\n" + DateTime.Now.ToString("MM-dd HH:mm:ss") + " 状态：" + strStatus);
                    rtb.Text = NewsSpider.Utility.PublicValue.GetMessage();

                    #endregion
                }
            }
            catch (Exception ex)
            {
                NewsSpider.Utility.Log.SaveErrorLog("获取网页地址列表 ->" + ex.Message);
            }
        }


        /// <summary>
        /// 采集文章内容
        /// </summary>
        public static void GetArticlePageContent()
        {
            NewsSpider.Utility.IniFile iniFile = new Utility.IniFile(System.Windows.Forms.Application.StartupPath + "\\config.ini");
            int iFlag = 1;
            try
            {
                string strLink = iniFile.GetString("CATCH", "LINK", "");
                string strTitleStartTag = iniFile.GetString("CATCH", "TitleStartTag", "");
                string strTitleEndTag = iniFile.GetString("CATCH", "TitleEndTag", "");
                string strContentStartTag = iniFile.GetString("CATCH", "ContentStartTag", "");
                string strContentEndTag = iniFile.GetString("CATCH", "ContentEndTag", "");
                string strAuthorStartTag = iniFile.GetString("CATCH", "AuthorStartTag", "");
                string strAuthorEndTag = iniFile.GetString("CATCH", "AuthorEndTag", "");
                string strSourceStartTag = iniFile.GetString("CATCH", "SourceStartTag", "");
                string strSourceEndTag = iniFile.GetString("CATCH", "strSourceEndTag", "");
                string strSiteName = iniFile.GetString("CATCH", "SiteName", "");
                string strCharset = iniFile.GetString("CATCH", "Charset", "");

                //抓取网页源代码
                NewsSpider.Utility.WebClient wc = new NewsSpider.Utility.WebClient();
                wc.Encoding = System.Text.Encoding.GetEncoding(strCharset);
                wc.OpenRead(strLink);
                string strGetPageHtmlCode = wc.RespHtml;

                string strGetTitle = "";
                string strGetContent = "";
                string strGetSource = "";
                string strGetAuthor = "";

                //采集文章标题
                try
                {
                    //Regex rxTitle = new Regex(strTitleStartTag + "(?<content>[\\s\\S]*)?" + strTitleEndTag, RegexOptions.Singleline);
                    Regex rxTitle = new Regex("<div class=\"blkContainerSblk\">(?<content>[\\s\\S]*)?</h1>", RegexOptions.Singleline);
                    strGetTitle = rxTitle.Match(strGetPageHtmlCode).Result("${content}");
                    strGetTitle = NewsSpider.Utility.StringHelper.CleanHtml(strGetTitle);
                    strGetTitle = strGetTitle.Trim();
                }
                catch (Exception ex)
                {
                    NewsSpider.Utility.Log.SaveErrorLog("文章标题采集失败：\r\n" + ex.Message + "," + strLink);
                    iFlag = 1;
                }

                if (strGetTitle != "")
                {
                    //采集文章内容
                    try
                    {
                        Regex rxContent = new Regex(strContentStartTag + "(?<content>[\\s\\S]*)?" + strContentEndTag, RegexOptions.Singleline);
                        strGetContent = rxContent.Match(strGetPageHtmlCode).Result("${content}");
                        strGetContent = NewsSpider.Utility.StringHelper.CleanHtml(strGetContent);
                        strGetContent = strGetContent.Trim();
                    }
                    catch (Exception ex)
                    {
                        NewsSpider.Utility.Log.SaveErrorLog("文章内容采集失败：\r\n" + ex.Message + "," + strLink);
                        iFlag = 2;
                    }

                    //采集文章来源
                    try
                    {
                        if (!string.IsNullOrEmpty(strSourceStartTag))
                        {
                            Regex rxSource = new Regex(strSourceStartTag + "(?<content>[\\s\\S]*)?" + strSourceEndTag, RegexOptions.Singleline);
                            strGetSource = rxSource.Match(strGetPageHtmlCode).Result("${content}");
                            strGetSource = NewsSpider.Utility.StringHelper.CleanHtml(strGetSource);
                            strGetSource = strGetSource.Trim();
                        }
                    }
                    catch (Exception ex)
                    {
                        NewsSpider.Utility.Log.SaveErrorLog("文章来源采集失败：\r\n" + ex.Message + "," + strLink);
                        iFlag = 3;
                    }

                    //采集文章作者
                    try
                    {
                        if (!string.IsNullOrEmpty(strAuthorStartTag))
                        {
                            Regex rxAuthor = new Regex(strAuthorStartTag + "(?<content>[\\s\\S]*)?" + strAuthorEndTag, RegexOptions.Singleline);
                            strGetAuthor = rxAuthor.Match(strGetPageHtmlCode).Result("${content}");
                            strGetAuthor = NewsSpider.Utility.StringHelper.CleanHtml(strGetAuthor);
                            strGetAuthor = strGetAuthor.Trim();
                        }
                    }
                    catch (Exception ex)
                    {
                        NewsSpider.Utility.Log.SaveErrorLog("文章来源采集失败：\r\n" + ex.Message + "," + strLink);
                        iFlag = 4;
                    }

                    try
                    {
                        NewsSpider.DAL.ArticlesSQLiteDAL asdal = new DAL.ArticlesSQLiteDAL();
                        if (!asdal.CheckArticleIsExist(strLink))
                        {
                            if (!string.IsNullOrEmpty(strGetContent))
                            {
                                strGetContent = strGetContent.Replace("try{if(window._S_uaTrack){_S_uaTrack('finance_stock_hotcol', 'pageview');}}catch(e){}", "");
                                strGetContent = strGetContent.Replace("'", "");

                                NewsSpider.Model.Articles article = new Model.Articles();
                                article.ArticleTitle = strGetTitle;
                                article.ArticleContent = strGetContent;
                                article.ArticleAuthor = strGetAuthor;
                                article.ArticleSource = strGetSource;
                                article.ArticleLink = strLink;

                                NewsSpider.Utility.IniFile iniFileData = new Utility.IniFile(Application.StartupPath + "\\data.ini");
                                string strFlag = iniFileData.GetString("Connection", "flag", "");

                                //存储到SQLite数据库
                                NewsSpider.BLL.ArticlesSQLiteBLL sqlitebll = new ArticlesSQLiteBLL();
                                sqlitebll.InsertArticlesBLL(article);

                                if (strFlag == "2")
                                {
                                    //存储到Access数据库
                                    NewsSpider.BLL.ArticlesAccessBLL accessbll = new ArticlesAccessBLL();
                                    accessbll.InsertArticlesBLL(article);
                                }
                                if (strFlag == "3")
                                {
                                    //存储到MSSQL数据库
                                    NewsSpider.BLL.ArticlesMSSQLBLL mssqlbll = new ArticlesMSSQLBLL();
                                    mssqlbll.InsertArticlesBLL(article);
                                }
                                if (strFlag == "4")
                                {
                                    //存储到MySQL数据库
                                    NewsSpider.BLL.ArticlesMySQLBLL mysqlbll = new ArticlesMySQLBLL();
                                    mysqlbll.InsertArticlesBLL(article);
                                }
                                if (strFlag == "5")
                                {
                                    //存储到Oracle数据库
                                    NewsSpider.BLL.ArticlesOracleBLL oraclebll = new ArticlesOracleBLL();
                                    oraclebll.InsertArticlesBLL(article);
                                }

                                iFlag = 0;
                            }
                            else
                            {
                                iFlag = 5;//内容为空
                            }
                        }
                        else
                        {
                            iFlag = 6;//文章标题重复
                        }
                    }
                    catch (Exception ex)
                    {
                        NewsSpider.Utility.Log.SaveErrorLog("添加数据失败：\r\n" + ex.Message + "," + strLink);
                        iFlag = 7;
                    }
                }
            }
            catch (Exception ex)
            {
                NewsSpider.Utility.Log.SaveErrorLog("根据文章链接获取文章对应的内容，并添加到数据库中 -> " + ex.Message);
                iFlag = 8;
            }

            iniFile.WriteValue("CATCH", "STATUSCODE", iFlag);
        }
    }
}
